# _*_coding:utf-8 _*_
# @Time? ? :2019/5/6 15:10
# @Author? :Dapan
# @Email : wali666@qq.com

"""从网站后台获取到的 搜索词，一个一个添加到 redis 缓存池中"""
import json
import redis

# redis 数据库配置
REDIS_HOST = "192.168.3.11"  # 主机名
REDIS_PORT = 6379  # 端口
REDIS_ENCODING = "utf8"  # 编码
# REDIS_PARAMS = {   # 指定数据库
#    'db':1
# }

class RedisQueue():
    def __init__(self):
        self.redisPool = redis.ConnectionPool(
            host=REDIS_HOST,
            port=REDIS_PORT,
            # db=1,
            decode_responses=True  # 加上decode_responses=True，写入的键值对中的value为str类型，不加这个参数写入的则为字节类型。
        )
        self.client = redis.StrictRedis(connection_pool=self.redisPool)
        # self.pipeline = self.client.pipeline()  # 创建管道

    def lpush_args(self, args_info):
        self.client = redis.Redis(connection_pool=self.redisPool)
        self.client.lpush("tb_spider:search_args", args_info)

    def brpop_args(self):
        self.client = redis.Redis(connection_pool=self.redisPool)
        search_args = self.client.brpop("tb_spider:search_args")
        print(search_args)
        print("search_args:", search_args[1])
        return search_args

    def lpush_url(self, uuid, keyword, data_sources, sort_type, page_num):
        self.client = redis.Redis(connection_pool=self.redisPool)
        url_temp = "https://s.taobao.com/search?q={}&sort={}&s={}"
        sort_types_ = {
            "综合排序": "default",
            "销量排序": "sale-desc",
            "信用由高到低": "credit-desc",
            "价格从低到高": "price-asc",
            "价格从高到低": "price-desc",
            "总价从低到高": "total-asc",
            "总价从高到低": "total-desc",
        }

        if data_sources == "PC":
            for i in range(page_num):
                # self.client.lpush("tb_spider:urls", url_temp.format(keyword, sort_types_[sort_type], i*44))
                self.client.lpush("tb_spider:urls-{}".format(uuid), url_temp.format(keyword, sort_type, i * 44))
        elif data_sources == "APP":
            pass  # TODO APP
        else:
            pass  # TODO H5

    def brpop_url(self):
        self.client = redis.Redis(connection_pool=self.redisPool)
        url = self.client.brpop("tb_spider:urls")
        print("url:", url[1])
        return url

    def lpush_page_data(self, data):  # Java使用后修改为 下边的rpush
        now_page_num = data['page_order']
        query = data['search_args']['keyword']
        data_sources = data['search_args']['data_sources']
        sort_type = data['search_args']['sort_type']
        redis_key = "tb_spider:cache_page_data:{}-{}-{}-{}".format(query, data_sources, sort_type, now_page_num)
        data = json.dumps(data)  # 将数据转换成json字符串插入redis
        self.client.lpush(redis_key, data)
        self.client.expire(redis_key, 3600)  # 有效期3600s

    def rpush_page_data(self, page_item_list):
        """
        将整页的数据以json字符串形式插入redis中
        :param page_item_list:
        :return:
        """
        # now_page_num = page_item_list[0]['page_order']
        print(page_item_list[0]['search_args'])
        query = page_item_list[0]['search_args']['keyword']
        data_sources = page_item_list[0]['search_args']['data_sources']
        sort_type = page_item_list[0]['search_args']['sort_type']
        now_page_num = page_item_list[0]['search_args']['now_page_num']
        redis_key = "tb_spider:cache_page_data:{}-{}-{}-{}".format(query, data_sources, sort_type, now_page_num)

        page_data_str = json.dumps(page_item_list)  # 将数据转换成json字符串插入redis
        self.client.rpush(redis_key, page_data_str)
        print(page_data_str)
        self.client.expire(redis_key, 3600)  # 有效期3600s

        # print("缓存数据插入成功", self.client.llen(redis_key))

    def lpush_item(self, item):
        print('00')
        search_uuid = item['search_uuid']
        print(search_uuid)
        print('01')
        print('02')
        data = json.dumps(item)  # 将数据转换成json字符串插入redis
        print('03')
        self.client.lpush("tb_spider:item:{}".format(search_uuid), data)
        print('04')

    def brpop_item(self, search_uuid):
        self.client = redis.Redis(connection_pool=self.redisPool)
        item = self.client.brpop("tb_spider:item_{}".format(search_uuid))
        print("url:", item[1])
        return item

    def set_page_data(self, key, page_data):
        """
        从redis队列中获取key对应的value，没有时返回none
        :param key:
        :return:
        """
        self.client = redis.Redis(connection_pool=self.redisPool)
        self.client.set('tb_spider:cache_page_data:{}'.format(key), page_data)

    def has_page_data(self, key):
        """
        判断页面缓存数据 的key的 列表长度
        :param key:
        :return:
        """
        # time.sleep(0.006) # 时间测试过，再小就可能查不到数据了
        self.client = redis.Redis(connection_pool=self.redisPool)
        result = self.client.llen('tb_spider:cache_page_data:{}'.format(key))
        if result != 0:
            print("结果：", result)
            return True
        else:
            print("结果：", result)
            return False

    def add_now_request_args(self, args_info):
        """
        给set集合中添加元素，标记为正在请求
        :param args_info:
        :return:
        """
        self.client = redis.Redis(connection_pool=self.redisPool)
        self.client.sadd("tb_spider:now_request_args", args_info)

    def remove_now_request_args(self, args_info):
        """
        根据key 和value 删除某个值
        :param args_info:
        :return:
        """
        self.client = redis.Redis(connection_pool=self.redisPool)
        print(self.client.srem("tb_spider:now_request_args", args_info))
        print("删除成功：", args_info)

    def check_args_st(self, args_info):
        """
        检测args 是不是已经在请求组里
        :param args_info:
        :return:
        """
        self.client = redis.Redis(connection_pool=self.redisPool)
        return self.client.sismember("tb_spider:now_request_args", args_info)


redis_queue = RedisQueue()

if __name__ == '__main__':
    # redis_queue.add_now_request_args({"q":11,"w":2})
    # redis_queue.add_now_request_args({"q":23})
    # redis_queue.add_now_request_args({"q":1})
    # redis_queue.remove_now_request_args({"q":11,"w":2})
    a = [{"search_args": {
        'sort_type': '_sale',
        'now_page_num': "1",
        'page_num_list': [1, 2],
        'keyword': '男装',
        'search_uuid': '123456',
        'data_sources': 'APP'
    }}]
    redis_queue.rpush_page_data(a * 1000000)
    print(redis_queue.has_page_data('男装-APP-_sale-1'))
